#! /usr/bin env python
# coding:utf-8
import json
import logging
import sys, getopt
import threading
import requests
import espider.app.jd.config as Config
import time
import espider.utils as Utils
from espider.app.jd.jd_exception import JdException
from espider.app.jd.jd_logic import JdLogic
from espider.spider import Spider
from lxml import etree


class JdSpider(threading.Thread, Spider):
    def __init__(self, kwargs):
        threading.Thread.__init__(self, target=self.work, kwargs=kwargs)
        self.logic = JdLogic()
        self.maxReqCount = 99999  # 每页最大记录条数
        self.maxPageCount = 200  # 最大页数
        self.s = requests.session()
        self.listUrl = "http://search.jd.com/Search?keyword=%s&page=%s"
        self.detailUrl = "http://item.jd.com/%s.html"
        self.headers = {
            'Connection': 'keep-alive',
            'Cache-Control': 'max-age=0',
            'Upgrade-Insecure-Requests': '1',
            'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/95.0.4638.69 Safari/537.36',
            'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'Accept-Language': 'zh-CN, zh;q = 0.9',
            'keep-alive': 'keep-alive',
            'Cookie': 'shshshfpb=wyWibk1AAlQeZzJSXPAE_wg; unpl=JF8EAKtnNSttUUIAAhMGExoVGw9UW1gBGURQa2cNV15QTVxQSwobQhB7XlVdXhRLFB9sYxRUWVNLVw4bAisSEXtdVV9dDE4SC2plNWRaWEIZRElPKxEQe11Vbl0PTBEEaGIAV1pdTFwDGQodFxRLWlZuXDhMFwpfVzVUVFlMVQEeABkXIEptVl9cCk4WBmtmAGQWNkoZBRwFHRUXTlhXWVgPQxEBZ2EAUF1fSWQEKwE; __jdv=122270672|direct|-|none|-|1673684615011; __jdu=1667664426469739745163; shshshfpa=1ccbaf98-1fbb-6066-5af0-b1f3fbb9d7d1-1673684615; areaId=1; qrsc=3; ipLoc-djd=1-2805-55650-0; __jda=122270672.1667664426469739745163.1667664426.1673684615.1674398119.4; __jdb=122270672.1.1667664426469739745163|4.1674398119; __jdc=122270672; shshshfp=626b9e50903dd169bdf1b967bfdb8f4d; shshshsID=e0f7b8f7b221a8238c0ab688518f5a3f_1_1674398119311; rkv=1.0; 3AB9D23F7A4B3C9B=ZENC4L6A6JI55KHF3WUJNERDRKPVGB7WN7XKLYSFNL3VFZCFZQ2FQEBYHF5IV3GIKWEGQ7ZTBYOMOWO2ATOSZGINDY'
        }
        self.detailHeaders = {
            'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9',
            'accept-encoding': 'gzip, deflate, br',
            'accept-language': 'zh-CN,zh;q=0.9',
            'cache-control': 'max-age=0',
            'cookie': 'shshshfpa=bd8fc98d-ed08-87e3-0b85-81ab2328d379-1628435301; shshshfpb=kto6VtGDfUbWjAWb5gO1OYg==; TrackID=1wIT8B6W-Kno8ocbH3dMQSmViuuD-__1jkF3R2UriIjdEFIHOjsJ1fcR5V7wH-K7a6w_6IbQnYrD4J0tdvNYRjQbDIVEzs2oYTrR5oxZJrPb_xXO1G9nEBgP3AB7dLnmO; jsavif=1; jsavif=1; shshshfp=2a1e352667f251c8691defac4271f59b; __jda=122270672.1674567918973602995944.1674567919.1674567919.1674567919.1; __jdc=122270672; __jdv=122270672|direct|-|none|-|1674567918973; areaId=1; ip_cityCode=2802; ipLoc-djd=1-72-55653-0; __jdu=1674567918973602995944; token=b88e26a38ae32fc875012635b3b6a512,2,930318; __tk=ilfxilaDnxPyWcaLnZyDVMBziZp0RUOwiMjKRUAERUAMWZoLRxOyiA,2,930318; shshshsID=044e77d6e8d85a8d00aaab4d8b0a27f2_10_1674573218506; __jdb=122270672.10.1674567918973602995944|1.1674567919; 3AB9D23F7A4B3C9B=A666YGK33IQNR5JEONBKW63HNH57EPWZSX7KCYU66SZCGGHGHH4QYOPAHQVVCRCDP2BTY4ANBXZHW4LOMZUYJNJYLE',
            'sec-ch-ua': '"Google Chrome";v="107", "Chromium";v="107", "Not=A?Brand";v="24"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': '"macOS"',
            'sec-fetch-dest': 'document',
            'sec-fetch-mode': 'navigate',
            'sec-fetch-site': 'same-origin',
            'sec-fetch-user': '?1',
            'upgrade-insecure-requests': '1',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'
        }
        self.priceHeaders = {
            'accept': '*/*',
            'accept-encoding': 'gzip, deflate, br',
            'accept-language': 'zh-CN,zh;q=0.9',
            'cookie': '__jdu=1659632771821422587826; shshshfpa=5d60a41d-1bcd-1799-5f6a-6a73293b613e-1666194205; shshshfpb=cTZTFywfni-Fbi21zW6OxDQ; __jdv=122270672|direct|-|none|-|1673709752002; areaId=1; ipLoc-djd=1-2805-55650-0; shshshfp=004d367716ebd3909320bafa12ce198f; __jdc=122270672; ip_cityCode=2802; 3AB9D23F7A4B3C9B=U5CULOH4TZ27HFEN3KQMIQLXIXX3DRIEMKWETH64ONYCF77RFEAHJR4XD3SWQMGSHWTCJCP3BZU6MD2NF446FJHOUU; token=e6e224bd0981d98e331b8ec57c7387e2,2,930309; __tk=AuexkVpp4ztskVGyjDex4VApAV4DAVAs4zs04zjyADe,2,930309; shshshsID=43f6082e79d89bebe2443a686da3900c_1_1674557751023; jsavif=1; __jda=122270672.1659632771821422587826.1659632771.1674550820.1674557752.10; __jdb=122270672.1.1659632771821422587826|10.1674557752',
            'referer': 'https://item.jd.com/',
            'sec-ch-ua': '"Google Chrome";v="107", "Chromium";v="107", "Not=A?Brand";v="24"',
            'sec-ch-ua-mobile': '?0',
            'sec-ch-ua-platform': 'Windows',
            'sec-fetch-dest': 'script',
            'sec-fetch-mode': 'no-cors',
            'sec-fetch-site': 'same-site',
            'user-agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36'
        }
        # self.keywords = ['小米手机', '华为手机', 'oppo', '荣耀手机', 'vivo', '红米', 'realme', '三星手机', '一加']
        self.keywords = ['红米', 'realme', '三星手机', '一加']
        self.startpage = 148
        self.exclude = ['红米']
        # init log
        self.logger = logging.getLogger('jdlog')
        self.logger.setLevel(logging.INFO)
        fh = logging.FileHandler("detail.csv")
        formatter = logging.Formatter('%(message)s')
        fh.setFormatter(formatter)
        self.logger.addHandler(fh)

    def work(self, **kwargs):
        operation = kwargs.get('operation', {})
        if 'getskulist' == operation:
            self.spiderSkuList()
        if 'saveskudata' == operation:
            filename = kwargs.get('filename', '')
            self.saveSkuData(filename)

    # 抓取商品列表页
    def spiderSkuList(self):
        try:
            for keyword in self.keywords:
                pages = range(self.maxPageCount)
                if keyword in self.exclude:
                    pages = pages[self.startpage:]
                for page in pages:
                    result = self.s.get(self.listUrl % (keyword, page), headers=self.headers).text
                    html = etree.HTML(result)
                    lidata = html.xpath('//*[@id="J_goodsList"]/ul/li')
                    if 0 == len(lidata):
                        break
                    self.logger.info("--%s--(%s)--" % (keyword, page))
                    for li in lidata:
                        sku = li.get('data-sku')
                        self.logger.info(self.detailUrl % sku)
                    time.sleep(Config.SLEEP_TIME)
        except JdException as e:
            self.log(e.code, e.msg, log_prefix="jd")

    # save
    def saveSkuData(self, filename):
        try:
            with open(filename, 'r') as logfile:
                for detailurl in logfile:
                    if '--' not in detailurl:
                        urlinfo = detailurl.split('item.jd.com/')
                        if len(urlinfo) <= 1:
                            continue
                        detailfile = urlinfo[1]
                        sku = detailfile.split('.')[0]
                        result = self.s.get(detailurl, headers=self.detailHeaders).text
                        html = etree.HTML(result)
                        # 京东价
                        # price = html.xpath('//div[contains(@class,"summary")]//span[contains(@class, "J-p-%s")]' % sku)[0].text
                        # price = self.getPrice(sku)
                        # 上市日期
                        releaseDate = html.xpath(
                            '//*[@id="detail"]//dt[contains(text(),"上市日期")]/following-sibling::dd[1]')
                        if len(releaseDate) > 0:
                            releaseDateText = releaseDate[0].text
                        else:
                            releaseDateText = '1970-01-01'
                        # 入网型号
                        netAccessModel = html.xpath(
                            '//*[@id="detail"]//dt[contains(text(),"入网型号")]/following-sibling::dd[2]')
                        if len(netAccessModel) > 0:
                            netAccessModelText = netAccessModel[0].text
                        else:
                            netAccessModelText = "未获取"
                        # 机身内存
                        fuselageMemory = html.xpath(
                            '//*[@id="detail"]//dt[contains(text(),"机身内存")]/following-sibling::dd[1]')
                        if len(fuselageMemory) > 0:
                            fuselageMemoryText = fuselageMemory[0].text
                        else:
                            fuselageMemory = html.xpath('//*[@id="detail"]//li[contains(text(),"机身内存")]')
                            if len(fuselageMemory) > 0:
                                fuselageMemoryArr = fuselageMemory[0].text.split('机身内存：')
                                if len(fuselageMemoryArr) <= 1:
                                    fuselageMemoryText = "未获取"
                                else:
                                    fuselageMemoryText = fuselageMemoryArr[1]
                            else:
                                fuselageMemoryText = "未获取"
                        # 运行内存
                        runningMemory = html.xpath(
                            '//*[@id="detail"]//dt[contains(text(),"运行内存")]/following-sibling::dd[1]')
                        if len(runningMemory) > 0:
                            runningMemoryText = runningMemory[0].text
                        else:
                            runningMemory = html.xpath('//*[@id="detail"]//li[contains(text(),"运行内存")]')
                            if len(runningMemory) > 0:
                                runningMemoryArr = runningMemory[0].text.split('运行内存：')
                                if len(runningMemoryArr) <= 1:
                                    runningMemoryText = "未获取"
                                else:
                                    runningMemoryText = runningMemoryArr[1]
                            else:
                                runningMemoryText = "未获取"
                        self.logger.info("%s,%s,%s,%s,%s,%s" % (
                        sku, releaseDateText, netAccessModelText, fuselageMemoryText, runningMemoryText,
                        detailurl.strip()))
                        time.sleep(Config.SLEEP_TIME)
        except JdException as e:
            self.log(e.code, e.msg, log_prefix="jd")

    # 获取价格
    # skuIds格式：J_000000,J_000001,J_000002
    def getPrice(self, skuIds):
        result = self.s.get('https://fts.jd.com/prices/mgets?skuIds=%s&source=pc-item' % skuIds,
                            headers=self.priceHeaders).text
        jsondata = json.loads(result)


if __name__ == "__main__":
    operation = ''
    argv = sys.argv[1:]
    try:
        opts, args = getopt.getopt(argv, "ho:f:", ["operation=", "file="])
    except getopt.GetoptError:
        print('python3 jd.py -o <operation> -f [file]')
        sys.exit(2)
    for opt, arg in opts:
        if opt == '-h':
            print('python jd.py -o <operation> -f [file]')
            sys.exit()
        elif opt in ("-o", "--operation"):
            operation = arg
        elif opt in ("-f", "--file"):
            filename = arg
    kwargs = {"operation": operation, "filename": filename}
    lw = JdSpider(kwargs)
    lw.start()
    lw.join()
